# -*-coding:utf-8-*-
import sys
import requests
from lxml import etree
from props.sql_connect import DBSession
from crawler_base import crawler_base
from DAO.tag import tag
from props.properties import props
from props.properties import get_header

reload(sys)
sys.setdefaultencoding('utf-8')


class tag_crawler(crawler_base):
    def tag_crawler_run(self,proxy):
        tagpage_url = props.get('tagpage.url')
        self.get_tag(tagpage_url,proxy)

    def get_tag(self, tag_page_url, proxy):
        count = 1
        tag_page = requests.get(tag_page_url, headers=get_header(),proxies=proxy).content.decode("utf-8", 'ignore')
        selector = etree.HTML(tag_page)
        all_tag = selector.xpath('//*[@id="content"]/div/div[1]/div[2]/div')
        session = DBSession()
        for i in range(len(all_tag)):
            father_tag = all_tag[i]
            father_tag_name = father_tag.xpath('./a/h2/text()')[0].replace('· · · · · ·', '').replace('\n', '').replace(
                ' ',
                '')
            father_id = count
            count += 1
            father_tag_instance = tag(father_tag_name, 0, -1)
            session.add(father_tag_instance)
            child_tags = father_tag.xpath('./table//td')
            print father_tag_name + ":"
            for j in range(len(child_tags)):
                count += 1
                child_tag_name = child_tags[j].xpath('./a/text()')[0].replace('\n', '').replace(' ', '')
                child_tag_instance = tag(child_tag_name, 1, father_id)
                session.add(child_tag_instance)
                print "   " + child_tag_name
            session.commit()

        session.close()
        print 'tag_end'

    def work(self, param=None, proxy=None):
        self.tag_crawler_run(proxy)
